package com.wz.spider;

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupExample {

	public static Set<String> checkedURL = new HashSet<String>();

	private static void doSelect() throws IOException{
        Document doc = Jsoup.connect("http://www.baimao.com/").get();
        Elements links = doc.select("a[href]");
        Elements media = doc.select("[src]");
        Elements imports = doc.select("link[href]");
        print("\nMedia: (%d)", media.size());
        for (Element src : media) {
            if (src.tagName().equals("img"))
                print(" * %s: <%s> %sx%s (%s)",
                        src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),
                        trim(src.attr("alt"), 20));
            else
                print(" * %s: <%s>", src.tagName(), src.attr("abs:src"));
        }
        print("\nImports: (%d)", imports.size());
        for (Element link : imports) {
            print(" * %s <%s> (%s)", link.tagName(),link.attr("abs:href"), link.attr("rel"));
        }
        print("\nLinks: (%d)", links.size());
        for (Element link : links) {
            print(" * a: <%s>  (%s)", link.attr("abs:href"), trim(link.text(), 35));
        }
	}

	private static void doSelect(Map<String, WebSite> urlMaps) throws IOException{
		Set<String> keys = urlMaps.keySet();
		Map<String, WebSite> selectedUrls = new HashMap<String, WebSite>();
		for (String urlName : keys) {
			WebSite web = urlMaps.get(urlName);
			System.out.println(web.url);
			Document doc = null;
			try {
				doc = Jsoup.connect(web.url).timeout(10000).get();
			} catch(Exception e){
				System.out.println("连接出现问题");
				continue;
			}
			Elements links = doc.select("a[href]");
			print("\nLinks: (%d)", links.size());
			for (Element link : links) {
				String selectedUrl = link.attr("abs:href");

				//	        	if (!selectedUrl.contains(urlName)){
				//	        		print(" * a: <%s>  (%s)", link.attr("abs:href"), trim(link.text(), 35));
				//	        	}

				if (!selectedUrl.contains(urlName) && !selectedUrl.contains("gov")&& selectedUrl!=null && selectedUrl.length()>0){
					WebSite webSite = new WebSite(getFirstURL(selectedUrl), trim(link.text(), 35));
					String name_url = getURLName(selectedUrl);
					if (selectedUrls.get(name_url)==null || (selectedUrls.get(name_url).name==null && selectedUrls.get(name_url).name.length()>0)){
						if (checkedURL.contains(name_url)){
							System.out.println("--已经搜集过此网址---" + urlName);
							continue;
						}
						checkedURL.add(name_url);
						System.out.println("已经收集:" + checkedURL.size() + "个网站");

						selectedUrls.put(name_url, webSite);
					}
				}
				//	            print(" * a: <%s>  (%s)", link.attr("abs:href"), trim(link.text(), 35));
			}

			Set<String> k = selectedUrls.keySet();
			for (String string : k) {
				System.out.println(string + "------" + selectedUrls.get(string).url + "---" + selectedUrls.get(string).name);
			}

			web.sons = selectedUrls;
			if (selectedUrls.size()==0){
				return;
			} else doSelect(selectedUrls);
		}
	}

	public static String getURLName(String url){
		try{
			if (url==null || url.length()<=0){
				return null;
			}
			String body = null;
			if (url.contains("http") || url.contains("https")){
				body = url.split("//")[1];
			}
			String domain = body.split("/")[0];
			String[] parts = domain.split("\\.");
			return parts[1];
		} catch(Exception e){
			System.out.println("url有误——" + url);
			return null;
		}

	}

	public static String getFirstURL(String url){
		if (url==null || url.length()<=0 || !url.contains("//")){
			return null;
		}
		String[] urlsParts = url.split("//");
		String body = null;
		if (url.contains("http") || url.contains("https")){
			body = urlsParts[1];
		}
		return urlsParts[0] + "//" + body.split("/")[0];
	}

    public static void main(String[] args) throws IOException {
    	Map<String, WebSite> urlMaps = new HashMap<String, WebSite>();
    	urlMaps.put("baimao", new WebSite("http://www.baimao.com/", "百贸"));
    	doSelect(urlMaps);
    }

    private static void print(String msg, Object... args) {
        System.out.println(String.format(msg, args));
    }
    private static String trim(String s, int width) {
        if (s.length() > width)
            return s.substring(0, width-1) + ".";
        else
            return s;
    }

    public static class WebSite{
    	String url;
    	String name;
    	Map<String, WebSite> sons = new HashMap<String, JsoupExample.WebSite>();
		public WebSite(String url, String name) {
			super();
			this.url = url;
			this.name = name;
		}
    }
}